"""
This module is responsible for creating the business rules for the CCHR
conversion system.
Exported Classes:
Exported Methods:
parse_rules -- This method is called during the stage and process phase to
create the business rules for that phase.
Exceptions:
RuleParsingError
ConvertError -- This error is raised when a value we are trying to convert
can not be handled by a Crosswalk object.
DataError
FatalDataError
"""
import os, imp
from itertools import count, izip
from datetime import datetime, date
from time import strptime
from itertools import izip
from sys import exc_info
import traceback
import cardsharp as cs
import log
from errors import *
from rules import *
from util import *
from configuration import config
from offense_rules import OffenseRules
from sentence_rules import SentenceRules
from admrec_rules import AdmrecRules
from region import *
from util import regions as util_regions
from metadata import *
import re
from segment import segment_dict
from contextlib import closing
import MySQLdb
from collections import defaultdict
_count = count().next
[docs]def mod_counter():
return 'bjs_' + str(_count())
#TODO rework log so it is created outside of VC
[docs]class VariableConverter(object):
def __init__(self, time, segment, var, phase, **kw):
self.func = None
self.name = var
self.id_count = None
self.regions = dict()
self.region_order = []
self.format_setters = set()
self.halt_on_error = kw.get('halt_on_error', False)
self.var = var
self.log = log.Log(time=time, segment=segment, var=var, phase=phase, log_dir=kw.get('log_dir'))
self.options = kw
[docs] def add_region(self, region, mod, crosswalk):
if mod:
self.format_setters.add(mod)
if crosswalk:
for key in crosswalk.iterkeys():
self.format_setters.add(crosswalk[key])
self.options.update({'region':region.get_state()[0]})
kw = self.options
self.regions[region] = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, **kw)
self.region_order.append(region)
self.region_order.sort(key = lambda r: r.priority())
[docs] def create_func(self, mod, crosswalk, add_info):
if mod:
self.format_setters.add(mod)
if crosswalk:
for key in crosswalk.iterkeys():
self.format_setters.add(crosswalk[key])
self.func = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, add_info, **self.options)
[docs] def set_auto_id(self, start_value=1):
#TODO put this rule in loader
"""This modules sets the translate function to pass an id value to the conversion function. The first id
will start at **start_value**, which has a default value of 0, and will increment by one for each row."""
self.id_count = count(start_value).next
[docs] def translate(self, state_var, value, row):
try:
#if regions are specified then loop over them and return the region specfic function
for region in self.region_order:
#TODO pass in map of rules to get 1 iteration over dataset
if row[state_var].lower() in region.get_state():
if self.id_count is not None:
return self.regions[region](self.id_count())
else:
return self.regions[region](value, row)
#no regions are specified than return global function
if self.id_count is not None:
return self.func(self.id_count())
else:
return self.func(value, row)
except:
if config.debug_halt_on_error:
raise
else:
self.log.write('fatal_error', ([str(exc_info()[:2])]), 99) #traceback.format_exc()
def _get_region_func(log, var, mod, crosswalk, error, add_info=None, **kw):
if mod and crosswalk:
mod.crosswalk = crosswalk
if mod:
#pass objects to mod
off_rules = OffenseRules(log)
sentence_rules = SentenceRules(log)
mod.log = log
mod.ConvertError = ConvertError
mod.RuleError = cs.rules.RuleError
mod.DataError = DataError
mod.FatalDataError = FatalDataError
mod.halt_on_error = error
mod.regions = regions
mod.key_as_str = key_as_str
mod.get_state_key = get_state_key
mod.default_offense_codes = off_rules.missing_codes
mod.o_vars = off_rules.vars
mod.o_default_map = off_rules.missing_map
mod.offense_not_coded = off_rules.not_coded_map
mod.convert_inc = off_rules.convert_inc
mod.convert_offenses = off_rules.convert_offenses
mod.convert_sentences = sentence_rules.convert_sentences
mod.sentence_missing = sentence_rules.missing
mod.sentence_unknown = sentence_rules.unknown
mod.sentence_rules = sentence_rules
mod.re = re
mod.izip = izip
mod.date = date
mod.strptime = strptime
mod.add_info=add_info
mod.math_rule = RegexMathRule
mod.datetime = datetime
mod.ma_duplicate = defaultdict(int)
mod.current_set = config.current_set
#TODO use regions instead of get_state_key
mod.regions = util_regions
mod.opt = kw
mod.get_cnxn = get_cnxn
#=======================================================================
# Define the function which the variable converter will return to the
# Cardsharp task
#=======================================================================
if var in ['ancic', 'cncic'] and kw.get('region'):
om = OffenseMetadata(var, 'arrest' if var[0] == 'a' else 'sentence', kw.get('region'))
#add a prediction rules attribute for the offense rules
mod.offense_rules = om.offense_map
if hasattr(mod, 'convert'):
def func(value, row):
try:
return mod.convert(value, row)
#handle type excpetion for easier debugging
except TypeError as err:
raise ConvertError("""Unable to call convert on var %s. for id:%s with value:%s. error was: %s""" % (var, row['id'], value, err))
return func
elif hasattr(mod, 'assign_vals'):
def func(value, row):
#TODO:snt_processor is deprecated remove from all modules
#snt_processor = SNTParser(row, mod.log)
return mod.assign_vals(value, row, None)
return func
elif hasattr(mod, 'translate'):
def func(value, row):
return mod.translate(value)
return func
elif hasattr(mod, 'assign_id'):
def func(value):
return mod.assign_id(value)
return func
elif hasattr(mod, 'validate'):
return mod.validate
elif hasattr(mod, 'cache'):
def func(value, row):
return mod.cache(row)
return func
elif crosswalk:
if var in ['ancic', 'cncic']:
def func(value, row):
vars = convert_offenses(row, crosswalk, offense_rules)
for k, v in vars.iteritems():
row[var[0]+k] = v
return vars['ncic']
#nonarr rules
elif var == 'nonarr':
admrec_meta = AdmrecMetadata(var, 'arrest', kw.get('region'))
admrec_rules = AdmrecRules(log, 'arrest', crosswalk, admrec_meta.admrec_map)
def func(value, row):
return admrec_rules.convert_admrec(row)
#noncrt rules
elif var == 'noncrt':
admrec_meta = AdmrecMetadata(var, 'sentence', kw.get('region'))
admrec_rules = AdmrecRules(log, 'court', crosswalk, admrec_meta.admrec_map)
def func(value, row):
return admrec_rules.convert_admrec(row)
else:
def func(key, value, row):
return crosswalk[key].convert(value)
return func
else:
raise RuleParsingError('error creating rule %s: kw:%s' % (var,str(kw)))
def _get_where_state_from_id(segment_name, **opt):
if 'id' in opt['in_where']:
where_ids = re.findall('id=(\d+)', opt['in_where'])
where_states = set()
try:
with closing(get_cnxn(opt['db_info']['name'],opt)) as cnx:
with closing(cnx.cursor()) as c:
for _id in where_ids:
c.execute("""SELECT rapstatex
FROM %sx
WHERE id=%s""" % (
segment_name,
_id)
)
where_states.update([get_state_key(c.fetchone()[0])])
except TypeError:
#we did not find the id in the dataset so return an empty set
return set()
return where_states
_crosswalk_region_re = re.compile('crosswalk_([a-zA-Z]{2})(\.txt|_\d+.txt)')
def _skip_crosswalk(where_regions, filename, verbose, region=None):
if where_regions and region == 'wvfbi':
crosswalk_region = _crosswalk_region_re.search(filename)
if crosswalk_region:
skip = True
for r in where_regions:
if crosswalk_region.groups()[0] == get_state_name(r):
skip = False
elif get_state_name(r) == 'ne' and crosswalk_region.groups()[0] == 'nb':
skip = False
if skip:
if verbose:
print 'skipping crosswalk %s' % filename
return True
return False
[docs]def parse_rules(rule_dir, segment, **opt):
db_info = opt['db_info']
phase = opt['phase_id']
segment_objs = opt['segments']
halt_on_error = opt['halt_on_err']
verbose = opt['verbose']
meta_dir = opt['meta_dir']
prefix = ''
xwalk_kw = {}
#opt['segment_name'] = segment['label']
#TODO: change db_info['name'] to 'filename' to allow db_info to be passed as keyword
time = datetime.now().isoformat('_').replace(':', '-')
_var_map, vars = {}, {}
variables = cs.load(source=os.path.join(meta_dir, segment['label'], 'var_info.xls'),
dataset='vars', format='excel')
cs.wait()
for row in variables:
_var_map[row['name'].lower()] = (row['name'], row['format'])
additional_data = {}
if re.search('pre_process_rules_3', rule_dir) and segment['label'] == 'sentence' and opt['phase_id'] != '3':
print 'loading cfed data...'
additional_data = {}
#TODO change this into mssql query to avoid slowness of cardsharp
options = {'source': db_info['name'], 'dataset': 'arrestx', 'format':'mysql',
'user': db_info['user'], 'pwd': db_info['pass'], 'load_as_null':[''],
'select': ['casenum', 'cycnum', 'afed', 'astate']}
if opt.get('in_where'):
options['where'] = opt.get('in_where')
ds = cs.load(**options)
cs.wait()
for row in ds:
additional_data['%s|%s' % (row['casenum'], row['cycnum'])] = (row['afed'], row['astate'])
cs.wait()
print 'complete.'
#for each segment
#loop over segment directories (arrest, demographic, sentence, supervision)
#set where_regions
where_regions = set()
if opt.get('in_where'):
where_regions = set(re.findall('state[ ]*=[ ]*(\d+)', opt['in_where']))
if re.search('id[ ]*=[ ]*',opt['in_where']):
_r = _get_where_state_from_id(segment['label'], **opt)
if _r and 'wvfbi' not in _r: where_regions.update(_r) #do not remove wvfbi crosswalks
for segment_dir in os.listdir(rule_dir):
segment_name = segment_dir.lower()
segment_dir = os.path.join(rule_dir, segment_dir)
if os.path.isfile(segment_dir):
continue #if file than not a segment directory, check next
elif segment_name != segment['label']:
continue #only load the segment we are runing rules for
#for each variable
#loop over all variable folders within a segment
for var_dir in os.listdir(segment_dir):
#if include rules are specified skip out of variables not in the include rules
if opt.get('include_rules') and var_dir.lower() not in opt['include_rules']:
continue
#if exclude rules are specified skip out of variables in the exclude rules
if opt.get('exclude_rules') and var_dir.lower() in opt['exclude_rules']:
continue
var_name = var_dir.lower()
var_dir = os.path.join(segment_dir, var_dir)
if os.path.isfile(var_dir):
continue #if file than not a variable directory, check next
#create a variable converter
vc = VariableConverter(time, segment_dict[segment_name]['id'], _var_map[var_name][0], phase, **opt)
vc_mod = None
vc_crosswalk = dict()
#for each rule or region
#loop over all files in the variable directory
for filename in os.listdir(var_dir):
full_path = os.path.join(var_dir, filename)
#load crosswalk rule
if filename.startswith('crosswalk'):
if _skip_crosswalk(where_regions, verbose, filename):
continue
if verbose:
print 'Loading crosswalk for %s' % filename
if filename[len(filename) - 4:len(filename)].lower() != '.txt':
raise RuleParsingError('file type must be .txt not %s' % filename[len(filename) - 4:len(filename)])
vc_crosswalk[filename[10:len(filename) - 4].lower()] = Crosswalk(full_path, db_info, segment_objs[segment_name], vc.log)
#load func rule
elif filename == 'func.py':
if verbose:
print 'Loading functions for %s' % var_name
vc_mod = imp.load_source(mod_counter(), full_path)
elif filename.endswith('.pyc'):
pass
#load region rules
elif os.path.isdir(full_path):
region = get_region(filename)
crosswalk = dict()
mod = None
#TODO make a state_where object that contains region names
#check the where to see if we should only load state specific rules
if where_regions:
skip_region = True
for r in where_regions:
if get_state_key(r) in region.state:
skip_region = False
if get_state_key('wvfbi') not in region.state and skip_region:
if verbose:
print 'skipping %s' % region
continue
#for each rule
for filename in os.listdir(full_path):
file_path = os.path.join(full_path, filename)
filename = filename.lower()
#if we are in a wvfbi state
#load region crosswalk
if filename.startswith('crosswalk'):
if _skip_crosswalk(where_regions, filename, verbose, region.state[0]):
continue
if verbose:
print 'Loading crosswalk for %s' % var_name
if filename[len(filename) - 4:len(filename)].lower() != '.txt':
raise RuleParsingError('file type must be .txt not %s' %
filename[len(filename) - 4:len(filename)])
crosswalk[filename[10:len(filename) - 4].lower()] = Crosswalk(file_path, db_info, segment_objs[segment_name], vc.log)
#load func rule
elif filename == 'func.py':
if verbose:
print 'Loading functions for %s:%s' % (var_name, region)
mod = imp.load_source(mod_counter(), file_path)
elif filename.endswith('.pyc'):
pass
else:
raise RuleParsingError('Region directory %s for var %s can only contain "crosswalk.txt", or "func.py" files. Found: %s' % (region, var_name, filename))
#add region rules to variable converter
#pass segment_name for log output
#region used for log and to store region func / crosswalk
#func is the function to be associated with the converter
#crosswalk is the crosswalk to be used by the converter (in addition to func or standalone)
vc.add_region(region, mod, crosswalk)
else:
raise RuleParsingError(' directory %s can only contain region directories, "func.py" files, or "crosswalk.txt" files. Found: %s' % (var_name, filename))
#create variable rule
vc.create_func(vc_mod, vc_crosswalk, additional_data)
vars[(segment_name, var_name)] = vc
return vars